# 番茄小说网小说拉取
import requests
from bs4 import BeautifulSoup
import os

path = 'http://www.fqxs.org/'
catalog = 'book/dilinhongmeng/'  # 目录地址
filepath = "E:/xiaoshuo/"
suffix = ".txt"
filename = ""


# 获取所有目录
def get_catalog_list():
    if not os.path.exists(filepath):
        os.mkdir(filepath)
    r = requests.get(path + catalog)
    soup = BeautifulSoup(r.text, "html.parser")
    filename = soup.find(attrs={"id": "info"}).find("h1").text
    if os.path.exists(filepath+filename+suffix):
        os.remove(filepath+filename+suffix)
    a_list = soup.find(attrs={"id": "list"}).find_all(name="a")
    print(len(a_list))
    a_list = a_list[15:]
    print(len(a_list))
    for page in a_list:
        get_html(page.get("href"))


# 获取html内容并写到文件
def get_html(p):
    r = requests.get(path + p)
    soup = BeautifulSoup(r.text, "html.parser")
    t = soup.find(attrs={"class": "bookname"}).find(name="h1").text
    print(t)
    content_list = soup.find(attrs={"id": "content"}).find_all(name="p")
    append_file(t, content_list)


def append_file(t, content_list):
    file = open(filepath+filename+suffix, "a+")
    file.write(t)
    file.write("\n")
    for ct in content_list:
        file.write(ct.text)
        file.write("\n")
    file.close()


if __name__ == "__main__":
    get_catalog_list()

